import requests
from lxml import etree


class QiuBai(object):

	def __init__(self):
<<<<<<< HEAD
		# self.url = "https://www.qiushibaike.com/text/page/{}/"
=======
		self.url = "https://www.qiushibaike.com/text/"
>>>>>>> 5691972e4bf091b9ef954647d6764691efe7bfca
		self.headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36"}

	def get_data(self):
		resp = requests.get(self.url)
		resp_str = resp.content.decode()
		return resp_str

	def parse_data(self, data):
		html = etree.HTML(data)
		div_list = html.xpath("//div[@class='col1 old-style-col1']/div")
		data_list = []
		for div in div_list:
			item = {}
<<<<<<< HEAD
			# index_i = div_list.index(div)
			item['content'] = ''
			content = [i.strip() for i in div.xpath(".//div[@class='content']/span/text()")]
			for i in content:
				item['content'] += i
			item['author'] = div.xpath(".//h2/text()")[0].strip()
			author_best = div.xpath(".//div[@class='cmtMain']/span/text()")[0] if len(div.xpath(".//div[@class='cmtMain']/span/text()"))>0 else None
			author_comment = div.xpath(".//div[@class='cmtMain']/div/text()")[0].strip() if len(div.xpath(".//div[@class='cmtMain']/div/text()"))>0 else None
			if author_best is not None:
				item['most_comment'] = author_best + author_comment
				data_list.append(item)
			else:
				data_list.append(item)
		return data_list

	def print_data(self, content_list):
		print(len(content_list))
		for content in content_list:
			print(content)
		# with open('test.txt', 'a+') as f:
		# 	for content in content_list:
		# 		f.write(str(content))

	def run(self):
		for i in range(1,14):
			self.url = "https://www.qiushibaike.com/text/page/{}/" .format(i)
			data = self.get_data()
			content_list = self.parse_data(data)
			self.print_data(content_list)
=======
			item['content'] = div.xpath("//div[@class='content']")[0]
			item['author'] = div.xpath("//h2/text()")[0]
			author_most = div.xpath("//span[@class='cmt-name']/text()")[0]
			author_comment = div.xpath("//div[@class='main-text']/text()")[0]
			item[author_most] = author_comment
			data_list.append(item)
		return data_list

	def print_data(self, content_list):
		for content in content_list:
			print(content)

	def run(self):
		data = self.get_data()
		content_list = self.parse_data(data)
		self.print_data(content_list)
>>>>>>> 5691972e4bf091b9ef954647d6764691efe7bfca

if __name__ == "__main__":
	qiubai = QiuBai()
	qiubai.run()